package com.chance.cc.crawler.development.bootstrap.bilibili;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.downloader.HttpConfig;
import com.chance.cc.crawler.core.downloader.proxy.Proxy;
import com.chance.cc.crawler.core.filter.FilterUtils;
import com.chance.cc.crawler.core.record.CrawlerRecord;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.development.utils.RedisReader;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang3.StringUtils;

import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.InputStreamReader;
import java.util.Arrays;
import java.util.List;

import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerRequestType.filter;
import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerRequestType.turnPageItem;
import static com.chance.cc.crawler.development.scripts.allfeild.AICCommonField.Tag_Site_Info;

/**
 * @Author: ZhaoHhuan on 2021/9/3 10:51
 * @Email: 18638575967@163.com
 * @Description:
 *      b站文章record放入redis中
**/
public class BilibiliArticleUrlListToRedisStart {
    public static final String domainId = "bilibili";
    private static final List<String> urlList = Arrays.asList("https://www.pcauto.com.cn/nation/2576/25767151.html");

    private static Proxy proxy = new Proxy();
    static {
        //代理配置
        proxy.setHost("http-dyn.abuyun.com");
        proxy.setPort(9020);
        proxy.setUsername("HL89Q19E86E2987D");
        proxy.setPassword("71F33D94CE5F7BF2");
    }
    HttpConfig httpConfig = HttpConfig.me(domainId);


    public static void main(String[] args) {
        createItemRecordToRedis();
    }

    //将文件中的地址生成record放入redis中
    private static void createItemRecordToRedis(){
        String siteBiz = "article-simple";
        String siteInfo = "simple-20210903";
        String path = "F:\\需求文件\\bilibili_article.txt";
        HttpConfig httpConfig = HttpConfig.me(domainId);
        httpConfig.setHttpSite(siteBiz);
        //push kv first
        RedisReader redisReader = new RedisReader("192.168.1.215",6379,4);
        try {
            BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(new FileInputStream(path), "UTF-8"));
//
            String readLine = null;
            while ((readLine = bufferedReader.readLine()) != null) {
                if(StringUtils.isBlank(readLine)){
                    continue;
                }
                String url = readLine;
                CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
                        .startPageRequest(domainId, turnPageItem)
                        .domain(domainId)
                        .httpUrl(url)
                        .releaseTime(System.currentTimeMillis())
                        .httpConfig(httpConfig)
                        .filter(CrawlerEnum.CrawlerRecordFilter.dateRange)
                        .addFilterInfo(FilterUtils.memoryFilterKeyInfo(domainId))
                        .addFilterInfo(FilterUtils.dateRangeFilterInfo(24 * 30,null))
                        .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                        .resultLabelTag(CrawlerEnum.CrawlerDataType.interaction)
                        .resultLabelTag(CrawlerEnum.CrawlerDataType.comment)
                        .proxy(proxy)
                        .build();
                requestRecord.tagsCreator().bizTags().addDomain(domainId);
                requestRecord.tagsCreator().bizTags().addSiteBiz(siteBiz);
                requestRecord.tagsCreator().bizTags().addCustomKV(Tag_Site_Info,siteInfo);

                //添加评论去重信息
                CrawlerRecord filterCrawlerRecord = new CrawlerRecord();
                filterCrawlerRecord.setFilter(CrawlerEnum.CrawlerRecordFilter.dateRange);
                filterCrawlerRecord.addFilterInfo(FilterUtils.memoryFilterKeyInfo(StringUtils.joinWith("-",filter,domainId,"comment")));
                filterCrawlerRecord.addFilterInfo(FilterUtils.dateRangeFilterInfo(24 * 30,null));
                requestRecord.tagsCreator().bizTags().addCustomKV("comment_record_filter_info", JSON.toJSONString(filterCrawlerRecord));



                String requestKey = requestRecord.getRecordKey();
                //确保key值唯一
                String shaKey = DigestUtils.sha1Hex(System.currentTimeMillis()+requestKey);

                redisReader.mapPush("crawler-bilibili-article-simple-queue_map",shaKey, JSON.toJSONString(requestRecord));
                redisReader.listPush("crawler-bilibili-article-simple-queue_list",shaKey);
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

}
